import time
import requests
import re
import os
import shutil

from util.logger import get_logger

logger = get_logger('豆瓣电影top250.log', __file__)


try:
    logger.info("启动成功")
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36 Edg/141.0.0.0'
    }
    #
    root = 'static/豆瓣电影'
    if os.path.exists(root):
        shutil.rmtree(root)
    os.makedirs(root)

    for i in range(0, 250, 25):
        # time.sleep(1)
        logger.debug(f"开始爬取第{(25+i)//25}页")
        url = f'https://movie.douban.com/top250?start={i}&filter='
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            # print(response.text)
            datas = re.findall(r'<div class="pic">.*?<em>(.*?)</em>.*?<a href="(.*?)">.*?<img width="100" alt="(.*?)" src="(.*?)">.*?</a>.*?</div>', response.text, re.S)
            for data in datas:
                num = data[0]
                detail_url = data[1]
                title = data[2]
                img_url = data[3]
                logger.info(f"第{num}个, {title}, {detail_url}")
                # 详情页简介获取
                logger.debug(f"开始爬取第{(25+i)//25}页,第{num}个, {title}的简介")
                detail_response = requests.get(detail_url, headers=headers)
                detail_info = re.search(r'<div class="bd">.*?<p data-clamp=".*?">(.*?)</p>.*?</div>', detail_response.text, re.S)
                if detail_info:
                    detail_info = detail_info.group(1).strip().split('<br />')
                    detail_text = ""
                    for text in detail_info:
                        detail_text += text.strip()
                else:
                    detail_info = re.search(r'<span property="v:summary">(.*?)</span>', detail_response.text, re.S)
                    detail_info = detail_info.group(1).strip().split('<br />')
                    detail_text = ""
                    for text in detail_info:
                        detail_text += text.strip()
                # 保存图片
                logger.debug(f"开始爬取第{(25 + i) // 25}页,第{num}个, {title}的图片")
                img_response = requests.get(img_url, headers=headers)
                with open(f'{root}/{title}.jpg', 'wb') as f:
                    f.write(img_response.content)
except Exception as e:
    logger.debug(e)

logger.info("爬取结束")





















# ua = UserAgent()
#
# headers = {
#     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36 Edg/141.0.0.0'
# }
# #
# root = 'static/豆瓣电影'
# if os.path.exists(root):
#     shutil.rmtree(root)
# os.makedirs(root)
#
# for i in range(0, 250, 25):
#     url = f'https://movie.douban.com/top250?start={i}&filter='
#     response = requests.get(url, headers=headers)
#     logging.info(f'{response.url, response.status_code}')
#     time.sleep(1)
#     if response.status_code == 200:
#         detail_urls = re.findall(r'<li>.*?<a href="(.*?)">.*?</li>', response.text, re.S)
#         for detail_url in detail_urls:
#             response = requests.get(detail_url, headers=headers)
#             detail_img = re.findall(r'<a class="sub-cover".*?<img src="(.*?)" />.*?</a>', response.text, re.S)
#             detail_title = re.findall(r'<div class="sub-title">(.*?)</div>', response.text, re.S)
#             detail_info = re.findall(r'<div class="bd">.*?<p data-clamp=".*?">\n\s+(.*?)<br/>(.*?)</p>.*?</div>', response.text, re.S)
#             print(detail_img, detail_title, detail_info)
#             img_response = requests.get(detail_img[0], headers=headers)
#             with open(f'{root}/{detail_title[0]}.jpg', 'wb') as f:
#                 f.write(img_response.content)
